package zolmain;

import db.MYSQLControl;
import model.ParameterModel;
import org.apache.http.ParseException;
import parse.Parse;
import utils.HttpJsoupUtil;

import java.util.ArrayList;
import java.util.List;

public class ParameterMain {
    public static List<ParameterModel> dataList = new ArrayList<>();

    public static void main(String[] args) throws ParseException, InterruptedException {
        //从数据库中取URL
        List<Object> urlList = MYSQLControl.getListOneBySQL("select product_url from product", "product_url");
        //待爬的队列
        HttpJsoupUtil util = new HttpJsoupUtil();
        for (int i = 0; i < urlList.size(); i++) {
            System.out.println(urlList.get(i).toString());
            //获取HTML
            String html = util.getHTML(urlList.get(i).toString());
            //解析数据
            Parse.getParData(urlList.get(i).toString(), html, dataList);
            //批量存储
            if (dataList.size() == 500) {
                MYSQLControl.executeInsert_Par(dataList);
                dataList.clear();
            }
            if ((i + 1) % 500 == 0) {
                //产生随机数
                int m = (int) (Math.random() * 2) + 1;
                Thread.sleep(m * 1000L);
            }
        }
        //不足批量的，单独存储
        MYSQLControl.executeInsert_Par(dataList);
    }

}